package com.guoj.webmagic.wm;

import org.apache.commons.collections.CollectionUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.JsonPathSelector;

import java.util.List;

/**
 * @作者: guoj
 * @日期: 2019/6/4 17:03
 * @描述: 抓取花田网的processor
 * --- 抓取的是：http://love.163.com/open/api/recommend/home/list
 * --- 这是个失败的案例，写法没问题，但是这个网站需要登录成功才能访问列表数据。
 */
public class HuatianPageProcessor implements PageProcessor {

	private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);

	private static final String LIST_URL = "http://love.163.com/open/api/recommend/home/list";

	@Override
	public void process(Page page) {


		// 判断是否是推荐列表url
		if (page.getUrl().regex(LIST_URL).match()) {
			// js渲染的用内部jsonpathSelector
			List<String> ids = new JsonPathSelector("$.list[*].user.url").selectList(page.getRawText());
			if (CollectionUtils.isNotEmpty(ids)) {
				for (String id : ids) {
					page.addTargetRequest("https://love.163.com/" + id);
				}
			}
		} else {
			page.putField("nickName", new JsonPathSelector("$.list.user.nickName").select(page.getRawText()));
			page.putField("creditLevel", new JsonPathSelector("$.list.user.creditLevel").select(page.getRawText()));
			page.putField("basicId", new JsonPathSelector("$.list.user.basicId").select(page.getRawText()));
			page.putField("sex", new JsonPathSelector("$.list.user.sex").select(page.getRawText()));
			page.putField("age", new JsonPathSelector("$.list.user.age").select(page.getRawText()));
			page.putField("height", new JsonPathSelector("$.list.user.height").select(page.getRawText()));
			page.putField("position", new JsonPathSelector("$.list.user.position").select(page.getRawText()));
		}

	}

	@Override
	public Site getSite() {
		return site;
	}

	public static void main(String[] args) {
		// 调试阶段开1个线程，执行阶段可以开多个。
		Spider.create(new HuatianPageProcessor()).addUrl(LIST_URL).thread(1).run();
	}
}
